{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "align-en-ms.py\r\n",
      "alignment-eflomal-eng_Latn-zsm_Latn.ipynb\r\n",
      "alignment-eflomal-eng_Latn-zsm_Latn.py\r\n",
      "alignment-eflomal-en-ms.ipynb\r\n",
      "alignment-en-ms.pickle\r\n",
      "alignment.pickle\r\n",
      "align-ms-en.py\r\n",
      "align.py\r\n",
      "all_test.txt\r\n",
      "all_train.txt\r\n",
      "all_val.txt\r\n",
      "augmented-eng_Latn-zsm_Latn-noisy-en-ms-dev-set-v1.json\r\n",
      "augmented-eng_Latn-zsm_Latn-noisy-en-ms-dev-set-v2.json\r\n",
      "augmented-eng_Latn-zsm_Latn-noisy-ms-en-dev-set-v1.json\r\n",
      "augmented-eng_Latn-zsm_Latn-noisy-ms-en-dev-set-v2.json\r\n",
      "augmented-eng_Latn-zsm_Latn-noisy-ms-en-dev-set-v3.json\r\n",
      "augmented-en-ms-v1.json\r\n",
      "augmented-en-ms-v2.json\r\n",
      "augmented-en-ms-v2-part2.json\r\n",
      "augmented-en-ms-v2-part3.json\r\n",
      "augmented-en-ms-v3.json\r\n",
      "augmented-en-ms-v3-part2.json\r\n",
      "augmented-ms-en-1.json\r\n",
      "augmented-ms-en-2.json\r\n",
      "augmented-ms-en-3.json\r\n",
      "augmented-ms-en-v2.json\r\n",
      "augmented-ms-en-v3.json\r\n",
      "augmented-ms-en-v4.json\r\n",
      "bjn_Latn-eng_Latn\r\n",
      "bjn_Latn-ind_Latn\r\n",
      "bjn_Latn-jav_Latn\r\n",
      "cnn\r\n",
      "eng_Latn.dev\r\n",
      "eng_Latn-zsm_Latn.forward\r\n",
      "eng_Latn-zsm_Latn.left\r\n",
      "eng_Latn-zsm_Latn.reverse\r\n",
      "eng_Latn-zsm_Latn-v2.left\r\n",
      "eng_Latn-zsm_Latn-v2.right\r\n",
      "en-ms.forward\r\n",
      "en-ms.reverse\r\n",
      "eval-en-ms-noisy-base.ipynb\r\n",
      "eval-noisy-base.ipynb\r\n",
      "export-base-austronesian.ipynb\r\n",
      "export.ipynb\r\n",
      "export-long-t5-local-base.ipynb\r\n",
      "export-small-austronesian.ipynb\r\n",
      "export-t5-base-noisy.ipynb\r\n",
      "export-t5-small-Copy1.ipynb\r\n",
      "export-t5-small.ipynb\r\n",
      "export-t5-small-noisy.ipynb\r\n",
      "export-t5-super-tiny.ipynb\r\n",
      "filtered-austronesian.ipynb\r\n",
      "filtered-bjn_Latn-eng_Latn\r\n",
      "filtered-bjn_Latn-eng_Latn.translated\r\n",
      "filtered-bjn_Latn-ind_Latn\r\n",
      "filtered-bjn_Latn-jav_Latn\r\n",
      "filtered-eng_Latn-jav_Latn00.splitted\r\n",
      "filtered-eng_Latn-jav_Latn00.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn00.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn01.splitted\r\n",
      "filtered-eng_Latn-jav_Latn01.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn01.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn02.splitted\r\n",
      "filtered-eng_Latn-jav_Latn02.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn02.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn03.splitted\r\n",
      "filtered-eng_Latn-jav_Latn03.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn03.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn04.splitted\r\n",
      "filtered-eng_Latn-jav_Latn04.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn04.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn05.splitted\r\n",
      "filtered-eng_Latn-jav_Latn05.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn05.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn06.splitted\r\n",
      "filtered-eng_Latn-jav_Latn06.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn06.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn07.splitted\r\n",
      "filtered-eng_Latn-jav_Latn07.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn07.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn08.splitted\r\n",
      "filtered-eng_Latn-jav_Latn08.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn08.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn09.splitted\r\n",
      "filtered-eng_Latn-jav_Latn09.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn09.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn10.splitted\r\n",
      "filtered-eng_Latn-jav_Latn10.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn10.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn11.splitted\r\n",
      "filtered-eng_Latn-jav_Latn11.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn11.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn12.splitted\r\n",
      "filtered-eng_Latn-jav_Latn12.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn12.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn13.splitted\r\n",
      "filtered-eng_Latn-jav_Latn13.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn13.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn14.splitted\r\n",
      "filtered-eng_Latn-jav_Latn14.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn14.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn15.splitted\r\n",
      "filtered-eng_Latn-jav_Latn15.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn15.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn16.splitted\r\n",
      "filtered-eng_Latn-jav_Latn16.splitted.pickle\r\n",
      "filtered-eng_Latn-jav_Latn16.splitted.translated\r\n",
      "filtered-eng_Latn-jav_Latn.ipynb\r\n",
      "filtered-ind_Latn-jav_Latn\r\n",
      "get-ms-text.ipynb\r\n",
      "ind_Latn.dev\r\n",
      "ind_Latn-jav_Latn.gz\r\n",
      "jav_Latn.dev\r\n",
      "mp.py\r\n",
      "ms.txt00.splitted\r\n",
      "ms.txt00.splitted.jw.pickle\r\n",
      "ms.txt00.splitted.jw.requested\r\n",
      "ms.txt00.splitted.pa.pickle\r\n",
      "ms.txt00.splitted.pa.requested\r\n",
      "ms.txt00.splitted.pickle\r\n",
      "ms.txt00.splitted.requested\r\n",
      "ms.txt00.splitted.ta.pickle\r\n",
      "ms.txt00.splitted.ta.requested\r\n",
      "ms.txt00.splitted.zh-CN.pickle\r\n",
      "ms.txt00.splitted.zh-CN.requested\r\n",
      "ms.txt01.splitted\r\n",
      "ms.txt01.splitted.jw.pickle\r\n",
      "ms.txt01.splitted.jw.requested\r\n",
      "ms.txt01.splitted.pa.pickle\r\n",
      "ms.txt01.splitted.pa.requested\r\n",
      "ms.txt01.splitted.pickle\r\n",
      "ms.txt01.splitted.requested\r\n",
      "ms.txt01.splitted.ta.pickle\r\n",
      "ms.txt01.splitted.ta.requested\r\n",
      "ms.txt01.splitted.zh-CN.pickle\r\n",
      "ms.txt01.splitted.zh-CN.requested\r\n",
      "ms.txt02.splitted\r\n",
      "ms.txt02.splitted.jw.pickle\r\n",
      "ms.txt02.splitted.jw.requested\r\n",
      "ms.txt02.splitted.pa.pickle\r\n",
      "ms.txt02.splitted.pa.requested\r\n",
      "ms.txt02.splitted.pickle\r\n",
      "ms.txt02.splitted.requested\r\n",
      "ms.txt02.splitted.ta.pickle\r\n",
      "ms.txt02.splitted.ta.requested\r\n",
      "ms.txt02.splitted.zh-CN.pickle\r\n",
      "ms.txt02.splitted.zh-CN.requested\r\n",
      "ms.txt03.splitted\r\n",
      "ms.txt03.splitted.jw.pickle\r\n",
      "ms.txt03.splitted.jw.requested\r\n",
      "ms.txt03.splitted.pa.pickle\r\n",
      "ms.txt03.splitted.pa.requested\r\n",
      "ms.txt03.splitted.pickle\r\n",
      "ms.txt03.splitted.requested\r\n",
      "ms.txt03.splitted.ta.pickle\r\n",
      "ms.txt03.splitted.ta.requested\r\n",
      "ms.txt03.splitted.zh-CN.pickle\r\n",
      "ms.txt03.splitted.zh-CN.requested\r\n",
      "ms.txt04.splitted\r\n",
      "ms.txt04.splitted.jw.pickle\r\n",
      "ms.txt04.splitted.jw.requested\r\n",
      "ms.txt04.splitted.pa.pickle\r\n",
      "ms.txt04.splitted.pa.requested\r\n",
      "ms.txt04.splitted.pickle\r\n",
      "ms.txt04.splitted.requested\r\n",
      "ms.txt04.splitted.ta.pickle\r\n",
      "ms.txt04.splitted.ta.requested\r\n",
      "ms.txt04.splitted.zh-CN.pickle\r\n",
      "ms.txt04.splitted.zh-CN.requested\r\n",
      "ms.txt05.splitted\r\n",
      "ms.txt05.splitted.jw.pickle\r\n",
      "ms.txt05.splitted.jw.requested\r\n",
      "ms.txt05.splitted.pa.pickle\r\n",
      "ms.txt05.splitted.pa.requested\r\n",
      "ms.txt05.splitted.pickle\r\n",
      "ms.txt05.splitted.requested\r\n",
      "ms.txt05.splitted.ta.pickle\r\n",
      "ms.txt05.splitted.ta.requested\r\n",
      "ms.txt05.splitted.zh-CN.pickle\r\n",
      "ms.txt05.splitted.zh-CN.requested\r\n",
      "ms.txt06.splitted\r\n",
      "ms.txt06.splitted.jw.pickle\r\n",
      "ms.txt06.splitted.jw.requested\r\n",
      "ms.txt06.splitted.pa.pickle\r\n",
      "ms.txt06.splitted.pa.requested\r\n",
      "ms.txt06.splitted.pickle\r\n",
      "ms.txt06.splitted.requested\r\n",
      "ms.txt06.splitted.ta.pickle\r\n",
      "ms.txt06.splitted.ta.requested\r\n",
      "ms.txt06.splitted.zh-CN.pickle\r\n",
      "ms.txt06.splitted.zh-CN.requested\r\n",
      "ms.txt07.splitted\r\n",
      "ms.txt07.splitted.jw.pickle\r\n",
      "ms.txt07.splitted.jw.requested\r\n",
      "ms.txt07.splitted.pa.pickle\r\n",
      "ms.txt07.splitted.pa.requested\r\n",
      "ms.txt07.splitted.pickle\r\n",
      "ms.txt07.splitted.requested\r\n",
      "ms.txt07.splitted.ta.pickle\r\n",
      "ms.txt07.splitted.ta.requested\r\n",
      "ms.txt07.splitted.zh-CN.pickle\r\n",
      "ms.txt07.splitted.zh-CN.requested\r\n",
      "ms.txt08.splitted\r\n",
      "ms.txt08.splitted.jw.pickle\r\n",
      "ms.txt08.splitted.jw.requested\r\n",
      "ms.txt08.splitted.pickle\r\n",
      "ms.txt08.splitted.requested\r\n",
      "ms.txt08.splitted.zh-CN.pickle\r\n",
      "ms.txt08.splitted.zh-CN.requested\r\n",
      "ms.txt09.splitted\r\n",
      "ms.txt09.splitted.jw.pickle\r\n",
      "ms.txt09.splitted.jw.requested\r\n",
      "ms.txt09.splitted.pickle\r\n",
      "ms.txt09.splitted.requested\r\n",
      "ms.txt09.splitted.zh-CN.pickle\r\n",
      "ms.txt09.splitted.zh-CN.requested\r\n",
      "ms.txt10.splitted\r\n",
      "ms.txt10.splitted.jw.pickle\r\n",
      "ms.txt10.splitted.jw.requested\r\n",
      "ms.txt10.splitted.pickle\r\n",
      "ms.txt10.splitted.requested\r\n",
      "ms.txt10.splitted.requested.requested\r\n",
      "ms.txt10.splitted.zh-CN.pickle\r\n",
      "ms.txt10.splitted.zh-CN.requested\r\n",
      "ms.txt11.splitted\r\n",
      "ms.txt11.splitted.jw.pickle\r\n",
      "ms.txt11.splitted.jw.requested\r\n",
      "ms.txt11.splitted.pickle\r\n",
      "ms.txt11.splitted.requested\r\n",
      "ms.txt11.splitted.zh-CN.pickle\r\n",
      "ms.txt11.splitted.zh-CN.requested\r\n",
      "ms.txt12.splitted\r\n",
      "ms.txt12.splitted.jw.pickle\r\n",
      "ms.txt12.splitted.jw.requested\r\n",
      "ms.txt12.splitted.pickle\r\n",
      "ms.txt12.splitted.requested\r\n",
      "ms.txt12.splitted.zh-CN.pickle\r\n",
      "ms.txt12.splitted.zh-CN.requested\r\n",
      "ms.txt13.splitted\r\n",
      "ms.txt13.splitted.jw.pickle\r\n",
      "ms.txt13.splitted.jw.requested\r\n",
      "ms.txt13.splitted.pickle\r\n",
      "ms.txt13.splitted.requested\r\n",
      "ms.txt13.splitted.zh-CN.pickle\r\n",
      "ms.txt13.splitted.zh-CN.requested\r\n",
      "ms.txt14.splitted\r\n",
      "ms.txt14.splitted.jw.pickle\r\n",
      "ms.txt14.splitted.jw.requested\r\n",
      "ms.txt14.splitted.pickle\r\n",
      "ms.txt14.splitted.requested\r\n",
      "ms.txt14.splitted.zh-CN.pickle\r\n",
      "ms.txt14.splitted.zh-CN.requested\r\n",
      "ms.txt15.splitted\r\n",
      "ms.txt15.splitted.jw.pickle\r\n",
      "ms.txt15.splitted.jw.requested\r\n",
      "ms.txt15.splitted.pickle\r\n",
      "ms.txt15.splitted.requested\r\n",
      "ms.txt15.splitted.zh-CN.pickle\r\n",
      "ms.txt15.splitted.zh-CN.requested\r\n",
      "ms.txt16.splitted\r\n",
      "ms.txt16.splitted.jw.pickle\r\n",
      "ms.txt16.splitted.jw.requested\r\n",
      "ms.txt16.splitted.pickle\r\n",
      "ms.txt16.splitted.requested\r\n",
      "ms.txt16.splitted.zh-CN.pickle\r\n",
      "ms.txt16.splitted.zh-CN.requested\r\n",
      "ms.txt17.splitted\r\n",
      "ms.txt17.splitted.pickle\r\n",
      "ms.txt17.splitted.requested\r\n",
      "ms.txt18.splitted\r\n",
      "ms.txt18.splitted.pickle\r\n",
      "ms.txt18.splitted.requested\r\n",
      "ms.txt19.splitted\r\n",
      "ms.txt19.splitted.pickle\r\n",
      "ms.txt19.splitted.requested\r\n",
      "ms.txt20.splitted\r\n",
      "ms.txt20.splitted.pickle\r\n",
      "ms.txt20.splitted.requested\r\n",
      "ms.txt21.splitted\r\n",
      "ms.txt21.splitted.pickle\r\n",
      "ms.txt21.splitted.requested\r\n",
      "ms.txt22.splitted\r\n",
      "ms.txt22.splitted.pickle\r\n",
      "ms.txt22.splitted.requested\r\n",
      "ms.txt23.splitted\r\n",
      "ms.txt23.splitted.pickle\r\n",
      "ms.txt23.splitted.requested\r\n",
      "ms.txt24.splitted\r\n",
      "ms.txt24.splitted.pickle\r\n",
      "ms.txt24.splitted.requested\r\n",
      "ms.txt25.splitted\r\n",
      "ms.txt25.splitted.pickle\r\n",
      "ms.txt25.splitted.requested\r\n",
      "ms.txt26.splitted\r\n",
      "ms.txt26.splitted.pickle\r\n",
      "ms.txt26.splitted.requested\r\n",
      "ms.txt27.splitted\r\n",
      "ms.txt28.splitted\r\n",
      "ms.txt29.splitted\r\n",
      "ms.txt30.splitted\r\n",
      "ms.txt31.splitted\r\n",
      "ms.txt32.splitted\r\n",
      "ms.txt33.splitted\r\n",
      "ms.txt34.splitted\r\n",
      "ms.txt35.splitted\r\n",
      "ms.txt36.splitted\r\n",
      "ms.txt37.splitted\r\n",
      "populate-cnn-dailymail.ipynb\r\n",
      "prepare-data-austronesian.ipynb\r\n",
      "prepare-data-flan.ipynb\r\n",
      "prepare-data.ipynb\r\n",
      "prepare-data-longer.ipynb\r\n",
      "prepare-data-noisy.ipynb\r\n",
      "prepare-eng_Latn-zsm_Latn.ipynb\r\n",
      "prepare-long-t5-local-base.ipynb\r\n",
      "r-id2.py\r\n",
      "r-id3.py\r\n",
      "r-id.py\r\n",
      "r-jw2.py\r\n",
      "r-jw3.py\r\n",
      "r-jw.py\r\n",
      "r-malaysia\r\n",
      "r-pa.py\r\n",
      "r-ta.py\r\n",
      "run_t5.py\r\n",
      "r-zh-cn.py\r\n",
      "shuffled-test-austronesian-10k.json\r\n",
      "shuffled-test-austronesian.json\r\n",
      "shuffled-test.json\r\n",
      "singlish.txt\r\n",
      "summarization.ipynb\r\n",
      "test-austronesian.json\r\n",
      "test.json\r\n",
      "test-longer.json\r\n",
      "test-noisy.json\r\n",
      "test-noisy-shuffled.json\r\n",
      "test-translated-cnn-daily.jsonl\r\n",
      "tmp_trainer\r\n",
      "train\r\n",
      "translate-cnn-dailymail-train.ipynb\r\n",
      "translate-cnn-dailymail-val.ipynb\r\n",
      "translated-test\r\n",
      "translated-val\r\n",
      "translate_en_ms_t5.py\r\n",
      "Untitled1.ipynb\r\n",
      "Untitled2.ipynb\r\n",
      "Untitled3.ipynb\r\n",
      "Untitled4.ipynb\r\n",
      "Untitled.ipynb\r\n",
      "val-translated-cnn-daily.jsonl\r\n",
      "zsm_Latn.dev\r\n"
     ]
    }
   ],
   "source": [
    "!ls ~/translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ms_zh_cn = sorted(glob('/home/husein/translation/ms.txt*.splitted.zh-CN.requested'))\n",
    "len(ms_zh_cn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('processed-ms-zh-cn.jsonl', 'w') as fopen_l:\n",
    "    for f in ms_zh_cn:\n",
    "        with open(f) as fopen:\n",
    "            for l in fopen:\n",
    "                l = json.loads(l)\n",
    "                try:\n",
    "                    left = l['src']\n",
    "                    right = l['r']['result']\n",
    "                    d = {\n",
    "                        'left': left,\n",
    "                        'right': right,\n",
    "                    }\n",
    "                    fopen_l.write(f'{json.dumps(d)}\\n')\n",
    "                except:\n",
    "                    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1433471 processed-ms-zh-cn.jsonl\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l processed-ms-zh-cn.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
